
from IPython.display import HTML
HTML('''<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js "></script><script>
code_show=true; 
function code_toggle() {
if (code_show){
$('div.jp-CodeCell > div.jp-Cell-inputWrapper').hide();
} else {
$('div.jp-CodeCell > div.jp-Cell-inputWrapper').show();
}
code_show = !code_show
} 
$( document ).ready(code_toggle);</script><form action="javascript:code_toggle()"><input type="submit" value="Click here to toggle on/off the raw code."></form>
''')


display(HTML('<center><blockquote class="twitter-tweet"><p lang="en" dir="ltr">Spiderverse had one of the largest teams of animators to *ever* work on an animated movie, so I can assure you that it didn’t steal anyone’s job. THIS is ethical use of AI. It’s not stealing from anyone, and it’s making the animators life easier by eliminating repetitive tasks. <a href="https://t.co/ObebDlMaP7">https://t.co/ObebDlMaP7</a></p>&mdash; JV (@javi_khoso) <a href="https://twitter.com/javi_khoso/status/1667463965532733440?ref_src=twsrc%5Etfw">June 10, 2023</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script></center>'))


display(HTML('<center><blockquote class="twitter-tweet"><p lang="en" dir="ltr">As a children’s author/illustrator, it is saddening to see these books bc, apart from the ethics of AI and stolen artwork, kids deserve better!!!<br><br>I’m tired of people who see kidlit as an easy get-rich-quick scheme and putting in the absolute minimal effort into their books <a href="https://t.co/I5WnlTiZgS">https://t.co/I5WnlTiZgS</a></p>&mdash; anoosha syed (@foxville_art) <a href="https://twitter.com/foxville_art/status/1602135206974345216?ref_src=twsrc%5Etfw">December 12, 2022</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script></center>'))


display(HTML('<center><blockquote class="twitter-tweet"><p lang="en" dir="ltr">this….this is fucking sad. I don’t know what to do <a href="https://t.co/X7VL9TpPkU">pic.twitter.com/X7VL9TpPkU</a></p>&mdash; Deb JJ Lee (@jdebbiel) <a href="https://twitter.com/jdebbiel/status/1601663197031075840?ref_src=twsrc%5Etfw">December 10, 2022</a></blockquote> <script async src="https://platform.twitter.com/widgets.js" charset="utf-8"></script></center>'))

$ conda create --name demistified jupyter
$ conda activate demistified
$ pip install -U -qq git+https://github.com/huggingface/diffusers.git
$ pip install -qq accelerate transformers ftfy


# Import the necessary libraries

# For your local environment, you will not need to set up os.environ (but i will)
import os
os.environ['XDG_CACHE_HOME'] = '/home/msds2023/sbarot/.cache'
os.environ['HUGGINGFACE_HUB_CACHE'] = '/home/msds2023/sbarot/.cache'

# Standard Python libraries
import argparse
import itertools
import math
import random
from glob import glob

# For data handling and visualization
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

# For model training and evaluation
import torch
import torch.nn.functional as F
import torch.utils.checkpoint
from torch.utils.data import Dataset

# For image and text pre-processing, accelerated training, connection to stable diffusion
import PIL
from accelerate import Accelerator
from accelerate.logging import get_logger
from accelerate.utils import set_seed
from diffusers import AutoencoderKL, DDPMScheduler, PNDMScheduler, StableDiffusionPipeline, UNet2DConditionModel
from diffusers.optimization import get_scheduler
from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker
from PIL import Image
from torchvision import transforms
from tqdm.auto import tqdm
from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer
from transformers import pipeline

# Hide warnings (may opt to keep)
import warnings
warnings.filterwarnings("ignore")

def image_grid(imgs, rows, cols):
    """Return a single image of imgs in a rows x column grid"""
    assert len(imgs) == rows*cols

    w, h = imgs[0].size
    grid = Image.new('RGB', size=(cols*w, rows*h))
    grid_w, grid_h = grid.size
    
    for i, img in enumerate(imgs):
        grid.paste(img, box=(i%cols*w, i//cols*h))
    return grid

def freeze_params(params):
    """Freeze all params in a model"""
    for param in params:
        param.requires_grad = False
        
# prompt templates for an textual inversion training on an object
imagenet_templates_small = [
    "a photo of a {}",
    "a rendering of a {}",
    "a cropped photo of the {}",
    "the photo of a {}",
    "a photo of a clean {}",
    "a photo of a dirty {}",
    "a dark photo of the {}",
    "a photo of my {}",
    "a photo of the cool {}",
    "a close-up photo of a {}",
    "a bright photo of the {}",
    "a cropped photo of a {}",
    "a photo of the {}",
    "a good photo of the {}",
    "a photo of one {}",
    "a close-up photo of the {}",
    "a rendition of the {}",
    "a photo of the clean {}",
    "a rendition of a {}",
    "a photo of a nice {}",
    "a good photo of a {}",
    "a photo of the nice {}",
    "a photo of the small {}",
    "a photo of the weird {}",
    "a photo of the large {}",
    "a photo of a cool {}",
    "a photo of a small {}",
]

# prompt templates for an textual inversion training on a style
imagenet_style_templates_small = [
    "a painting in the style of {}",
    "a rendering in the style of {}",
    "a cropped painting in the style of {}",
    "the painting in the style of {}",
    "a clean painting in the style of {}",
    "a dirty painting in the style of {}",
    "a dark painting in the style of {}",
    "a picture in the style of {}",
    "a cool painting in the style of {}",
    "a close-up painting in the style of {}",
    "a bright painting in the style of {}",
    "a cropped painting in the style of {}",
    "a good painting in the style of {}",
    "a close-up painting in the style of {}",
    "a rendition in the style of {}",
    "a nice painting in the style of {}",
    "a small painting in the style of {}",
    "a weird painting in the style of {}",
    "a large painting in the style of {}",
]

# Class for Dataset
class TextualInversionDataset(Dataset):
    def __init__(
        self,
        data_root,
        tokenizer,
        learnable_property="object",  # [object, style]
        size=512,
        repeats=100,
        interpolation="bicubic",
        flip_p=0.5,
        set="train",
        placeholder_token="*",
        center_crop=False,
    ):

        self.data_root = data_root
        self.tokenizer = tokenizer
        self.learnable_property = learnable_property
        self.size = size
        self.placeholder_token = placeholder_token
        self.center_crop = center_crop
        self.flip_p = flip_p

        self.image_paths = [os.path.join(self.data_root, file_path) for file_path in os.listdir(self.data_root)]

        self.num_images = len(self.image_paths)
        self._length = self.num_images

        if set == "train":
            self._length = self.num_images * repeats

        self.interpolation = {
            "linear": PIL.Image.LINEAR,
            "bilinear": PIL.Image.BILINEAR,
            "bicubic": PIL.Image.BICUBIC,
            "lanczos": PIL.Image.LANCZOS,
        }[interpolation]

        self.templates = imagenet_style_templates_small if learnable_property == "style" else imagenet_templates_small
        self.flip_transform = transforms.RandomHorizontalFlip(p=self.flip_p)

    def __len__(self):
        return self._length

    def __getitem__(self, i):
        example = {}
        image = Image.open(self.image_paths[i % self.num_images])

        if not image.mode == "RGB":
            image = image.convert("RGB")

        placeholder_string = self.placeholder_token
        text = random.choice(self.templates).format(placeholder_string)

        example["input_ids"] = self.tokenizer(
            text,
            padding="max_length",
            truncation=True,
            max_length=self.tokenizer.model_max_length,
            return_tensors="pt",
        ).input_ids[0]

        # default to score-sde preprocessing
        img = np.array(image).astype(np.uint8)

        if self.center_crop:
            crop = min(img.shape[0], img.shape[1])
            h, w, = (
                img.shape[0],
                img.shape[1],
            )
            img = img[(h - crop) // 2 : (h + crop) // 2, (w - crop) // 2 : (w + crop) // 2]

        image = Image.fromarray(img)
        image = image.resize((self.size, self.size), resample=self.interpolation)

        image = self.flip_transform(image)
        image = np.array(image).astype(np.uint8)
        image = (image / 127.5 - 1.0).astype(np.float32)

        example["pixel_values"] = torch.from_numpy(image).permute(2, 0, 1)
        return example
    
def create_dataloader(train_batch_size=1):
    """Return a shuffled dataloader for a given train_datast"""
    return torch.utils.data.DataLoader(train_dataset, batch_size=train_batch_size, shuffle=True)


logger = get_logger(__name__)

def save_progress(text_encoder, placeholder_token_id, accelerator, save_path):
    """Checkpoints and saves latest learned embeddings"""
    logger.info("Saving embeddings")
    learned_embeds = accelerator.unwrap_model(text_encoder).get_input_embeddings().weight[placeholder_token_id]
    learned_embeds_dict = {placeholder_token: learned_embeds.detach().cpu()}
    torch.save(learned_embeds_dict, save_path)

def training_function(text_encoder, vae, unet):
    """Textual Inversion training function that needs to be passed to the accelerator"""
    train_batch_size = hyperparameters["train_batch_size"]
    gradient_accumulation_steps = hyperparameters["gradient_accumulation_steps"]
    learning_rate = hyperparameters["learning_rate"]
    max_train_steps = hyperparameters["max_train_steps"]
    output_dir = hyperparameters["output_dir"]
    gradient_checkpointing = hyperparameters["gradient_checkpointing"]

    accelerator = Accelerator(
        gradient_accumulation_steps=gradient_accumulation_steps,
        mixed_precision=hyperparameters["mixed_precision"]
    )

    if gradient_checkpointing:
        text_encoder.gradient_checkpointing_enable()
        unet.enable_gradient_checkpointing()

    train_dataloader = create_dataloader(train_batch_size)

    if hyperparameters["scale_lr"]:
        learning_rate = (
            learning_rate * gradient_accumulation_steps * train_batch_size * accelerator.num_processes
        )

    # Initialize the optimizer
    optimizer = torch.optim.AdamW(
        text_encoder.get_input_embeddings().parameters(),  # only optimize the embeddings
        lr=learning_rate,
    )

    text_encoder, optimizer, train_dataloader = accelerator.prepare(
        text_encoder, optimizer, train_dataloader
    )

    weight_dtype = torch.float32
    if accelerator.mixed_precision == "fp16":
        weight_dtype = torch.float16
    elif accelerator.mixed_precision == "bf16":
        weight_dtype = torch.bfloat16

    # Move vae and unet to device
    vae.to(accelerator.device, dtype=weight_dtype)
    unet.to(accelerator.device, dtype=weight_dtype)

    # Keep vae in eval mode as we don't train it
    vae.eval()
    # Keep unet in train mode to enable gradient checkpointing
    unet.train()

    
    # We need to recalculate our total training steps as the size of the training dataloader may have changed.
    num_update_steps_per_epoch = math.ceil(len(train_dataloader) / gradient_accumulation_steps)
    num_train_epochs = math.ceil(max_train_steps / num_update_steps_per_epoch)

    # Train!
    total_batch_size = train_batch_size * accelerator.num_processes * gradient_accumulation_steps

    logger.info("***** Running training *****")
    logger.info(f"  Num examples = {len(train_dataset)}")
    logger.info(f"  Instantaneous batch size per device = {train_batch_size}")
    logger.info(f"  Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}")
    logger.info(f"  Gradient Accumulation steps = {gradient_accumulation_steps}")
    logger.info(f"  Total optimization steps = {max_train_steps}")
    # Only show the progress bar once on each machine.
    progress_bar = tqdm(range(max_train_steps), disable=not accelerator.is_local_main_process)
    progress_bar.set_description("Steps")
    global_step = 0

    for epoch in range(num_train_epochs):
        text_encoder.train()
        for step, batch in enumerate(train_dataloader):
            with accelerator.accumulate(text_encoder):
                # Convert images to latent space
                latents = vae.encode(batch["pixel_values"].to(dtype=weight_dtype)).latent_dist.sample().detach()
                latents = latents * 0.18215

                # Sample noise that we'll add to the latents
                noise = torch.randn_like(latents)
                bsz = latents.shape[0]
                # Sample a random timestep for each image
                timesteps = torch.randint(0, noise_scheduler.num_train_timesteps, (bsz,), device=latents.device).long()

                # Add noise to the latents according to the noise magnitude at each timestep
                # (this is the forward diffusion process)
                noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps)

                # Get the text embedding for conditioning
                encoder_hidden_states = text_encoder(batch["input_ids"])[0]

                # Predict the noise residual
                noise_pred = unet(noisy_latents, timesteps, encoder_hidden_states.to(weight_dtype)).sample

                 # Get the target for loss depending on the prediction type
                if noise_scheduler.config.prediction_type == "epsilon":
                    target = noise
                elif noise_scheduler.config.prediction_type == "v_prediction":
                    target = noise_scheduler.get_velocity(latents, noise, timesteps)
                else:
                    raise ValueError(f"Unknown prediction type {noise_scheduler.config.prediction_type}")

                loss = F.mse_loss(noise_pred, target, reduction="none").mean([1, 2, 3]).mean()
                accelerator.backward(loss)

                # Zero out the gradients for all token embeddings except the newly added
                # embeddings for the concept, as we only want to optimize the concept embeddings
                if accelerator.num_processes > 1:
                    grads = text_encoder.module.get_input_embeddings().weight.grad
                else:
                    grads = text_encoder.get_input_embeddings().weight.grad
                # Get the index for tokens that we want to zero the grads for
                index_grads_to_zero = torch.arange(len(tokenizer)) != placeholder_token_id
                grads.data[index_grads_to_zero, :] = grads.data[index_grads_to_zero, :].fill_(0)

                optimizer.step()
                optimizer.zero_grad()

            # Checks if the accelerator has performed an optimization step behind the scenes
            if accelerator.sync_gradients:
                progress_bar.update(1)
                global_step += 1
                if global_step % hyperparameters["save_steps"] == 0:
                    save_path = os.path.join(output_dir, f"learned_embeds-step-{global_step}.bin")
                    save_progress(text_encoder, placeholder_token_id, accelerator, save_path)

            logs = {"loss": loss.detach().item()}
            progress_bar.set_postfix(**logs)

            if global_step >= max_train_steps:
                break

        accelerator.wait_for_everyone()


    # Create the pipeline using using the trained modules and save it.
    if accelerator.is_main_process:
        pipeline = StableDiffusionPipeline.from_pretrained(
            pretrained_model_name_or_path,
            text_encoder=accelerator.unwrap_model(text_encoder),
            tokenizer=tokenizer,
            vae=vae,
            unet=unet,
        )
        pipeline.save_pretrained(output_dir)
        # Also save the newly trained embeddings
        save_path = os.path.join(output_dir, f"learned_embeds.bin")
        save_progress(text_encoder, placeholder_token_id, accelerator, save_path)

2023-06-12 13:57:03.256015: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.


# Load the images
original_images = sorted(glob('data/Vibes/*.png'))
original_image_files = []
for original_image in original_images:
    original_image_files.append(Image.open(original_image).resize((256, 256)))

image_grid(original_image_files, 1, len(original_image_files))


pretrained_model_name_or_path = "stabilityai/stable-diffusion-2"
what_to_teach = "style"
placeholder_token = "<stevie-barot>"
initializer_token = "painting" 
save_path = './data/Vibes'

hyperparameters = {
    "learning_rate": 5e-04,
    "scale_lr": True,
    "max_train_steps": 2000,
    "save_steps": 250,
    "train_batch_size": 4,
    "gradient_accumulation_steps": 1,
    "gradient_checkpointing": True,
    "mixed_precision": "fp16",
    "seed": 42,
    "output_dir": "vibes-concept-output"
}

# !mkdir -p vibes-concept-output


# ~1.5MB
# Load the tokenizer and add the placeholder token as a additional special token.
tokenizer = CLIPTokenizer.from_pretrained(
    pretrained_model_name_or_path,
    subfolder="tokenizer",
)

# Add the placeholder token in tokenizer
num_added_tokens = tokenizer.add_tokens(placeholder_token)
if num_added_tokens == 0:
    raise ValueError(
        f"The tokenizer already contains the token {placeholder_token}. Please pass a different"
        " `placeholder_token` that is not already in the tokenizer."
    )
    
#Get token ids for our placeholder and initializer token
# This code block will complain if initializer string is not a single token

# Convert the initializer_token, placeholder_token to ids
token_ids = tokenizer.encode(initializer_token, add_special_tokens=False)
# Check if initializer_token is a single token or a sequence of tokens
if len(token_ids) > 1:
    raise ValueError("The initializer token must be a single token.")

initializer_token_id = token_ids[0]
placeholder_token_id = tokenizer.convert_tokens_to_ids(placeholder_token)


#Load the Stable Diffusion model, ~4.5GB
# Load models and create wrapper for stable diffusion

# pipeline = StableDiffusionPipeline.from_pretrained(pretrained_model_name_or_path)
# del pipeline

text_encoder = CLIPTextModel.from_pretrained(
    pretrained_model_name_or_path, subfolder="text_encoder"
)
vae = AutoencoderKL.from_pretrained(
    pretrained_model_name_or_path, subfolder="vae"
)
unet = UNet2DConditionModel.from_pretrained(
    pretrained_model_name_or_path, subfolder="unet"
)


text_encoder.resize_token_embeddings(len(tokenizer))
token_embeds = text_encoder.get_input_embeddings().weight.data
token_embeds[placeholder_token_id] = token_embeds[initializer_token_id]


# Freeze vae and unet
freeze_params(vae.parameters())
freeze_params(unet.parameters())
# Freeze all parameters except for the token embeddings in text encoder
params_to_freeze = itertools.chain(
    text_encoder.text_model.encoder.parameters(),
    text_encoder.text_model.final_layer_norm.parameters(),
    text_encoder.text_model.embeddings.position_embedding.parameters(),
)
freeze_params(params_to_freeze)


# Prepare dataset
train_dataset = TextualInversionDataset(
      data_root=save_path,
      tokenizer=tokenizer,
      size=vae.config.sample_size,
      placeholder_token=placeholder_token,
      repeats=100,
      learnable_property=what_to_teach,
      center_crop=False,
      set="train")


# 345B
noise_scheduler = DDPMScheduler.from_config(pretrained_model_name_or_path, subfolder="scheduler")


# import accelerate

# accelerate.notebook_launcher(training_function, num_processes=1, args=(text_encoder, vae, unet))

# for param in itertools.chain(unet.parameters(), text_encoder.parameters()):
#     if param.grad is not None:
#         del param.grad  # free some memory
#     torch.cuda.empty_cache()


# Set up the pipeline 
from diffusers import DPMSolverMultistepScheduler

pipe = StableDiffusionPipeline.from_pretrained(
    "vibes-concept-output",
    scheduler=DPMSolverMultistepScheduler.from_pretrained("vibes-concept-output", subfolder="scheduler"),
    torch_dtype=torch.float16,
).to("cuda")


prompts =  ["an image in the style of <stevie-barot>",
         "a misty forest with coniferous trees in the style of <stevie-barot>",
         "a warm purple sunset in the style of <stevie-barot>",
         "a plate of stacked pancakes with maple syrup in the style of <stevie-barot>",
         "a glass of red wine in the style of <stevie-barot>",
         "a room with the shadow of a window with a pink sunset in the style of <stevie-barot>",
         "a rocky sea shore with high waves in the style of <stevie-barot>",
         "a laser printer in the style of <stevie-barot>",
         "a paint brush painting a canvas with yellow paint in the style of <stevie-barot>"
        ]


# #Run the Stable Diffusion pipeline

# num_samples = 4
# num_rows = 1

# all_prompt_images = {}

# for prompt in prompts: 
#     all_images = []
#     for _ in range(num_rows):
#         images = pipe([prompt] * num_samples, num_inference_steps=30, guidance_scale=7.5).images
#         all_images.extend(images)
#     all_prompt_images[prompt] = all_images
    
# !mkdir -p data/Vibes_generated

# for p, prompt_images in enumerate(list(all_prompt_images.values())):
#     for i, im in enumerate(prompt_images):
#         im.save(f'data/Vibes_generated/{p}_{i}.png')


prompt_dict = {i: p for i, p in enumerate(prompts)}
all_prompt_images = {p: [] for p in prompts}

for i in glob('data/Vibes_generated/*.png'):
    p = int(os.path.basename(i)[0])
    all_prompt_images[prompt_dict[p]].append(Image.open(i))

original_dict = {i: j for i, j in zip([7, 8, 6, 3, 1, 5, 2, 4], original_images)}
original_dict = dict(zip([prompt_dict[o] for o in original_dict], list(original_dict.values())))

for prompt in list(all_prompt_images.keys())[1:]:
    fig, ax = plt.subplots(1, 2, figsize=(10,5), dpi=100)
    ax[0].imshow(Image.open(original_dict[prompt]))
    ax[0].set_xticks([])
    ax[0].set_yticks([])
    ax[1].imshow(image_grid(all_prompt_images[prompt], 2, 2))
    ax[1].set_xticks([])
    ax[1].set_yticks([])
    fig.suptitle(f'"{prompt}""')
    fig.tight_layout()
    plt.show()


original_misted_images = sorted(glob('data/Vibes_misted/*.png'))
original_misted_image_files = []
for original_image in original_misted_images:
    original_misted_image_files.append(Image.open(original_image).resize((256, 256)))
print('original images')
display(image_grid(original_image_files, 1, len(original_image_files)))
print('misted images')
display(image_grid(original_misted_image_files, 1, len(original_misted_image_files)))

original images

misted images


pretrained_model_name_or_path = "stabilityai/stable-diffusion-2"
what_to_teach = "style" # object or style
placeholder_token = "<stevie-misted>" # used to represent your new concept
initializer_token = "painting" #word that can summarise what your new concept is, to be used as a starting point
save_path = './data/Vibes_misted'

hyperparameters = {
    "learning_rate": 5e-04,
    "scale_lr": True,
    "max_train_steps": 4000,
    "save_steps": 250,
    "train_batch_size": 2,
    "gradient_accumulation_steps": 1,
    "gradient_checkpointing": True,
    "mixed_precision": "fp16",
    "seed": 42,
    "output_dir": "vibes-misted-concept-output"
}

# !mkdir -p vibes-misted-concept-output


# ~1.5MB
# Load the tokenizer and add the placeholder token as a additional special token.
tokenizer = CLIPTokenizer.from_pretrained(
    pretrained_model_name_or_path,
    subfolder="tokenizer",
)

# Add the placeholder token in tokenizer
num_added_tokens = tokenizer.add_tokens(placeholder_token)
if num_added_tokens == 0:
    raise ValueError(
        f"The tokenizer already contains the token {placeholder_token}. Please pass a different"
        " `placeholder_token` that is not already in the tokenizer."
    )
    
#Get token ids for our placeholder and initializer token
# This code block will complain if initializer string is not a single token

# Convert the initializer_token, placeholder_token to ids
token_ids = tokenizer.encode(initializer_token, add_special_tokens=False)
# Check if initializer_token is a single token or a sequence of tokens
if len(token_ids) > 1:
    raise ValueError("The initializer token must be a single token.")

initializer_token_id = token_ids[0]
placeholder_token_id = tokenizer.convert_tokens_to_ids(placeholder_token)

#Load the Stable Diffusion model, ~4.5GB
# Load models and create wrapper for stable diffusion

# pipeline = StableDiffusionPipeline.from_pretrained(pretrained_model_name_or_path)
# del pipeline

text_encoder = CLIPTextModel.from_pretrained(
    pretrained_model_name_or_path, subfolder="text_encoder"
)
vae = AutoencoderKL.from_pretrained(
    pretrained_model_name_or_path, subfolder="vae"
)
unet = UNet2DConditionModel.from_pretrained(
    pretrained_model_name_or_path, subfolder="unet"
)

text_encoder.resize_token_embeddings(len(tokenizer))
token_embeds = text_encoder.get_input_embeddings().weight.data
token_embeds[placeholder_token_id] = token_embeds[initializer_token_id]

# Freeze vae and unet
freeze_params(vae.parameters())
freeze_params(unet.parameters())
# Freeze all parameters except for the token embeddings in text encoder
params_to_freeze = itertools.chain(
    text_encoder.text_model.encoder.parameters(),
    text_encoder.text_model.final_layer_norm.parameters(),
    text_encoder.text_model.embeddings.position_embedding.parameters(),
)
freeze_params(params_to_freeze)

train_dataset = TextualInversionDataset(
      data_root=save_path,
      tokenizer=tokenizer,
      size=vae.config.sample_size,
      placeholder_token=placeholder_token,
      repeats=100,
      learnable_property=what_to_teach, #Option selected above between object and style
      center_crop=False,
      set="train")

# 345B
noise_scheduler = DDPMScheduler.from_config(pretrained_model_name_or_path, subfolder="scheduler")


# # may need to reduce batch_size and increase max_train_steps to allocate space for GPU
# import accelerate

# accelerate.notebook_launcher(training_function, num_processes=1, args=(text_encoder, vae, unet))

# for param in itertools.chain(unet.parameters(), text_encoder.parameters()):
#     if param.grad is not None:
#         del param.grad  # free some memory
#     torch.cuda.empty_cache()


# Set up the pipeline -- may need to restart to reallocate cuda
from diffusers import DPMSolverMultistepScheduler

pipe = StableDiffusionPipeline.from_pretrained(
    "vibes-misted-concept-output",
    scheduler=DPMSolverMultistepScheduler.from_pretrained(
        "vibes-misted-concept-output", subfolder="scheduler"),
    torch_dtype=torch.float16,
).to("cuda")


misted_prompts = ["an image in the style of <stevie-misted>",
                  "a misty forest with coniferous trees in the style of <stevie-misted>",
                  "a warm purple sunset in the style of <stevie-misted>",
                  "a plate of stacked pancakes with maple syrup in the style of <stevie-misted>",
                  "a glass of red wine in the style of <stevie-misted>",
                  "a room with the shadow of a window with a pink sunset in the style of <stevie-misted>",
                  "a rocky sea shore with high waves in the style of <stevie-misted>",
                  "a laser printer in the style of <stevie-misted>",
                  "a paint brush painting a canvas with yellow paint in the style of <stevie-misted>"
                  ]


# #Run the Stable Diffusion pipeline
# num_samples = 4
# num_rows = 1

# all_misted_prompt_images = {}

# for prompt in misted_prompts: 
#     all_images = []
#     for _ in range(num_rows):
#         images = pipe([prompt] * num_samples, num_inference_steps=30, guidance_scale=7.5).images
#         all_images.extend(images)
#     all_misted_prompt_images[prompt] = all_images
    
# !mkdir -p data/Vibes_misted_generated

# for p, prompt_images in enumerate(list(all_misted_prompt_images.values())):
#     for i, im in enumerate(prompt_images):
#         im.save(f'data/Vibes_misted_generated/{p}_{i}.png')


original_images = glob('data/Vibes_misted/*.png')
misted_prompt_dict = {i: p for i, p in enumerate(misted_prompts)}
all_misted_prompt_images = {p: [] for p in misted_prompts}

for i in glob('data/Vibes_misted_generated/*.png'):
    p = int(os.path.basename(i)[0])
    all_misted_prompt_images[misted_prompt_dict[p]].append(Image.open(i))

original_misted_dict = {i: j for i, j in zip([7, 1, 5, 6, 3, 2, 4, 8], original_images)}
original_misted_dict = dict(zip([misted_prompt_dict[o] for o in original_misted_dict],
                                list(original_misted_dict.values())))

for prompt in list(all_misted_prompt_images.keys())[1:]:
    fig, ax = plt.subplots(1, 2, figsize=(10,5), dpi=100)
    ax[0].imshow(Image.open(original_misted_dict[prompt]))
    ax[0].set_xticks([])
    ax[0].set_yticks([])
    ax[1].imshow(image_grid(all_misted_prompt_images[prompt], 2, 2))
    ax[1].set_xticks([])
    ax[1].set_yticks([])
    fig.suptitle(f'"{prompt}""')
    fig.tight_layout()
    plt.show()


# 331MB
ai_detector = pipeline('image-classification', model='umm-maybe/AI-image-detector')

Could not find image processor class in the image processor config or the model config. Loading based on pattern matching with the model's feature extractor configuration.


accuracies = {}
original_scores = []

# Detecting Original Art
for original_image in original_images:
    original_score = ai_detector(original_image)
    prediction = original_score[0]['label']
    prediction_score = original_score[0]['score']
    original_scores.append([original_image, prediction, prediction_score])
    
original_df = pd.DataFrame(original_scores, columns=['filepath', 'prediction', 'score'])
original_df['score'] = (original_df['score'] * 100).round(1)
original_df['illuminarty prediction'] = ['human'] * 8
original_df['illuminarty score'] = [100-x for x in [0.2, 0.0, 0.5, 1.5, 0.0, 0.1, 1.0, 0.2]]

original_preds = len(original_df[original_df['prediction'] == 'human'])
illuminarty_original_preds = len(original_df[original_df['illuminarty prediction'] == 'human'])
original_df['type'] = ['Original Art'] * len(original_df)

correct = (original_preds / len(original_df)) * 100
ill_correct = (illuminarty_original_preds / len(original_df)) * 100

# print(f"{correct:.2f}% Correct")
# print(f"{ill_correct:.2f}% Illuminarty Correct")

accuracies['Original Art umm-maybe'] = correct
accuracies['Original Art Illuminarty'] = ill_correct

# Detecting Artificial Art
artificial_images = glob('data/Vibes_generated/*.png')
artificial_scores = []

for artificial_image in artificial_images:
    artificial_score = ai_detector(artificial_image)
    prediction = artificial_score[0]['label']
    prediction_score = artificial_score[0]['score']
    artificial_scores.append([artificial_image, prediction, prediction_score])

artificial_df = pd.DataFrame(artificial_scores, columns=['filepath', 'prediction', 'score'])

artificial_preds = len(artificial_df[artificial_df['prediction'] == 'artificial'])

illuminarty_artificial_score = [86.3, 93.0, 95.1, 89.9,
                                58.2, 90.7, 64.5, 84.5,
                                88.8, 98.4, 89.3, 87.5,
                                46.7, 37.1, 87.3, 82.5,
                                71.1, 25.9, 84.1, 60.2,
                                97.1, 73.5, 40.9, 77.8,
                                42.3, 68.2, 64.4, 19.2,
                                48.7, 91.88, 75.3, 99.2,
                                76.3, 80.2, 91.8, 40.7]

illuminarty_prediction = ['artificial' if i > 50.0 else 'human' for i in illuminarty_artificial_score]
illuminarty_score = [i if i > 50.0 else 100-i for i in illuminarty_artificial_score]

artificial_df['illuminarty prediction'] = illuminarty_prediction
artificial_df['illuminarty score'] = illuminarty_score
artificial_df['type'] = ['Artificial Art'] * len(artificial_df)

illuminarty_artificial_preds = len(artificial_df[artificial_df['illuminarty prediction'] == 'artificial'])
correct = (artificial_preds / len(artificial_df)) * 100
ill_correct = (illuminarty_artificial_preds / len(artificial_df)) * 100

accuracies['Artificial Art umm-maybe'] = correct
accuracies['Artificial Art Illuminarty'] = ill_correct

# print(f"{correct:.2f}% Correct")
# print(f"{ill_correct:.2f}% Illuminarty Correct")

# Detecting Misted Art
misted_images = glob('data/Vibes_misted/*.png')

misted_scores = []

for misted_image in misted_images:
    misted_score = ai_detector(misted_image)
    prediction = misted_score[0]['label']
    prediction_score = misted_score[0]['score']
    misted_scores.append([misted_image, prediction, prediction_score])
    
misted_df = pd.DataFrame(misted_scores, columns=['filepath', 'prediction', 'score'])
misted_df['score'] = (misted_df['score'] * 100).round(1)
misted_df['illuminarty prediction'] = ['human'] * 8
misted_df['illuminarty score'] = [100-x for x in [0.2, 0.0, 0.5, 1.5, 0.0, 0.1, 1.0, 0.2]]
misted_df['type'] = ['Misted Art'] * len(misted_df)

misted_preds = len(misted_df[misted_df['prediction'] == 'human'])
illuminarty_misted_preds = len(misted_df[misted_df['illuminarty prediction'] == 'human'])

correct = (misted_preds / len(misted_df)) * 100
ill_correct = (illuminarty_misted_preds / len(misted_df)) * 100

# print(f"{correct:.2f}% Correct")
# print(f"{ill_correct:.2f}% Illuminarty Correct")

accuracies['Misted Art umm-maybe'] = correct
accuracies['Misted Art Illuminarty'] = ill_correct

# Detecting Artificial Misted Art
artificial_misted_images = glob('data/Vibes_misted_generated/*.png')
artificial_misted_scores = []

for artificial_image in artificial_misted_images:
    artificial_score = ai_detector(artificial_image)
    prediction = artificial_score[0]['label']
    prediction_score = artificial_score[0]['score']
    artificial_misted_scores.append([artificial_image, prediction, prediction_score])

artificial_misted_df = pd.DataFrame(artificial_misted_scores, columns=['filepath', 'prediction', 'score'])

artificial_misted_preds = len(artificial_misted_df[artificial_misted_df['prediction'] == 'artificial'])

illuminarty_artificial_misted_score = [100.0, 82.7, 83.2, 65.2,
                                       92.6, 17.6, 87.0, 69.6,
                                       99.4, 97.0, 72.9, 93.2,
                                       67.9, 14.2, 26.1, 81.6,
                                       16.6, 33.6, 3.8, 29.3,
                                       91.1, 82.6, 82.9, 93.8,
                                       51.7, 29.8, 77.4, 30.4,
                                       38.9, 81.5, 97.9, 64.4,
                                       56.8, 2.5, 92.9, 24.1
                                      ]

illuminarty_misted_prediction = ['artificial' if i > 50.0 else 'human' for i in illuminarty_artificial_misted_score]
illuminarty_misted_score = [i if i > 50.0 else 100-i for i in illuminarty_artificial_misted_score]

artificial_misted_df['illuminarty prediction'] = illuminarty_misted_prediction
artificial_misted_df['illuminarty score'] = illuminarty_misted_prediction
artificial_misted_df['type'] = ['Artificial Misted Art'] * len(artificial_misted_df)

correct = (artificial_misted_preds / len(artificial_misted_df)) * 100
illuminarty_artificial_misted_preds = len(artificial_misted_df[artificial_misted_df['illuminarty prediction'] == 'artificial'])
ill_correct = (illuminarty_artificial_misted_preds / len(artificial_misted_df)) * 100

accuracies['Artificial Misted Art umm-maybe'] = correct
accuracies['Aritificial Misted Art Illuminarty'] = ill_correct

# print(f"{correct:.2f}% Correct")
# print(f"{ill_correct:.2f}% Illuminarty Correct")


fig, ax = plt.subplots(figsize=(12, 6), dpi=100)
pd.DataFrame(accuracies, index=['Accuracies']).T.sort_values('Accuracies').plot.barh(
    title='AI Image Detector Accuracy per art type (in percent)',
    color='#8975F0',
    ax=ax
)
ax.set_xlim([0, 100]);


fig, ax = plt.subplots(figsize=(12, 6), dpi=100)

df = pd.concat([original_df, artificial_df, misted_df, artificial_misted_df])
df['actual'] = ['human' if (i == 'Original Art') or (i == 'Misted Art') else 'artificial' for i in df['type']]

correct = len(df[df['actual'] == df['prediction']]) / len(df)
ill_correct = len(df[df['actual'] == df['illuminarty prediction']]) / len(df)

pd.DataFrame({'umm-maybe accuracy': correct * 100, 'illuminarty accuracy': ill_correct * 100},
             index=['Accuracies']).T.sort_values('Accuracies').plot.barh(title='Accuracy per method (in percent)',
                                                                         color='#8975F0',
                                                                         ax=ax
                                                                        )

ax.set_xlim([0, 100]);


artificial_images = np.array_split(sorted(glob('data/Vibes_generated/*.png')), 9)
artificial_image_files = []
for artificial_array in artificial_images:
    artificial_array_files = []
    for artificial_image in artificial_array:
        artificial_array_files.append(Image.open(artificial_image).resize((256, 256)))
    artificial_image_files.append(artificial_array_files)
artificial_image_files = [image_grid(a, 2, 2) for a in artificial_image_files]
display(image_grid(np.array(artificial_image_files)[[7, 8, 6, 3, 1, 5, 2, 4]], 1, 8))

artificial_misted_images = np.array_split(sorted(glob('data/Vibes_misted_generated/*.png')), 9)
artificial_misted_image_files = []
for artificial_misted_array in artificial_misted_images:
    artificial_misted_array_files = []
    for artificial_misted_image in artificial_misted_array:
        artificial_misted_array_files.append(Image.open(artificial_misted_image).resize((256, 256)))
    artificial_misted_image_files.append(artificial_misted_array_files)
artificial_misted_image_files = [image_grid(a, 2, 2) for a in artificial_misted_image_files]
display(image_grid(np.array(artificial_misted_image_files)[[7, 8, 6, 3, 1, 5, 2, 4]], 1, 8))


fig, ax = plt.subplots(1, 2, figsize=(10,5), dpi=100)
ax[0].imshow(image_grid(all_prompt_images['an image in the style of <stevie-barot>'], 2, 2))
ax[0].set_title('<stevie-barot>')
ax[0].set_xticks([])
ax[0].set_yticks([])

ax[1].imshow(image_grid(all_misted_prompt_images['an image in the style of <stevie-misted>'], 2, 2))
ax[1].set_title('<stevie-misted>')
ax[1].set_xticks([])
ax[1].set_yticks([])

fig.suptitle('"an image in the style of..."')
fig.tight_layout()
plt.show()


detected_artificial_images = artificial_df[artificial_df['prediction'] == 'artificial']['filepath'].to_list()
detected_artificial_images = [Image.open(artificial_image).resize((256, 256)) for artificial_image in detected_artificial_images]
print('What umm-maybe correctly detected as artificial')
display(image_grid(detected_artificial_images, 1, 6))

detected_artificial_images = artificial_df[artificial_df['illuminarty prediction'] == 'human']['filepath'].to_list()
detected_artificial_images = [Image.open(artificial_image).resize((256, 256)) for artificial_image in detected_artificial_images]
print('What Illuminarty incorrectly detected as human')
display(image_grid(detected_artificial_images, 1, 8))

What umm-maybe correctly detected as artificial

What Illuminarty incorrectly detected as human

friend	vibe
el	yellow brush strokes on a canvas
cyrus	the cyan cartridge of a laser printer
madeline	a dim room illuminated by the light of a pink sky
kuggs	a misty forest filled with coniferous trees
yana	a glass of dark red wine
margot	a warm purple sunset in an empty field
juice	golden syrup poured on top of a stack of pankcakes
jara	a rocky sea shore with crashing waves on an overcast sky

The Context

If AI is so helpful, why do (most) artists hate it?

Little misunderstandings are what start wars

The threat of obsolescence

The act of theft

The feeling of helplessness

The Problem

The Gameplan

Textual Inversion with Stable Diffusion

Art Protection with Mist

Art Detection with umm-maybe and Illuminarty

Setting Up

Coding Environment

Dataset¶

Original Art to Artificial Art

Training

Generated Image Results

Original Art to Misted Art

Misted Art to Misted Artificial Art

Training¶

Generated Image Results¶

Art Detection

So Does it Work?

Mist is relatively successful in disrupting textual inversion in Stable Diffusion

Illuminarty is relatively good at detecting AI art

Conclusion

References